# 参考教程： https://www.cnblogs.com/Chary/p/18830285


# 三台机器
sudo hostnamectl set-hostname "k8smaster"      
sudo hostnamectl set-hostname "k8snode1"     
sudo hostnamectl set-hostname "k8snode2"    


cat >> /etc/hosts << EOF
192.168.5.30 k8smaster
192.168.5.31 k8snode1
192.168.5.32 k8snode2
EOF
# 后期修改ip ： 
sed -i 's/192.168.5.30/192.168.219.132/g' /etc/hosts
sed -i 's/192.168.5.31/192.168.219.133/g' /etc/hosts
sed -i 's/192.168.5.32/192.168.219.134/g' /etc/hosts
# systemctl restart containerd kubelet


cat << EOF | tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

cat << EOF | tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF


modprobe overlay
modprobe br_netfilter

sysctl --system


# 负载均衡工具
apt install -y ipset ipvsadm


cat << EOF | tee /etc/modules-load.d/ipvs.conf
ip_vs
ip_vs_rr
ip_VS_wrr
ip_vs_sh
nf_conntrack
EOF



cat << EOF | tee ipvs.sh
#!/bin/sh
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF


sh ipvs.sh

# 查看内核模块是否加载
lsmod | grep ip_vs

# 使内核模块配置参数生效
sysctl --system


# 关闭swap内存交换功能
sudo swapoff -a



# 安装containerd
wget https://github.com/containerd/containerd/releases/download/v1.7.27/cri-containerd-cni-1.7.27-linux-amd64.tar.gz
tar -zxvf cri-containerd-cni-1.7.27-linux-amd64.tar.gz -C /
echo "export PATH=$PATH:/usr/local/bin" >> ~/.bashrc
mkdir /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml
vim /etc/containerd/config.toml
  # 修改 sandbox_image 值为 registry.aliyuncs.com/google_containers/pause:3.9
  # 修改 SystemdCgroup 为 true


mkdir -p /etc/containerd/certs.d/docker.io
tee /etc/containerd/certs.d/docker.io/hosts.toml <<EOF
[host."https://docker.1ms.run"]
  capabilities = ["pull", "resolve"]
EOF

vim /etc/containerd/config.toml # 启用动态镜像配置目录：
[plugins."io.containerd.grpc.v1.cri".registry]
  config_path = "/etc/containerd/certs.d" 

sudo systemctl enable --now containerd



# 安装crictl ， 注意crictl与containerd版本
mkdir crictl && cd crictl
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.28.0/crictl-v1.28.0-linux-amd64.tar.gz
sudo tar zxvf crictl-v1.28.0-linux-amd64.tar.gz -C /usr/local/bin


cat <<EOF | tee /etc/crictl.yaml
runtime-endpoint: "unix:///var/run/containerd/containerd.sock"
image-endpoint: "unix:///var/run/containerd/containerd.sock"
timeout: 10
debug: false
pull-image-on-create: true
disable-pull-on-run: false
EOF

# 可以拉取镜像了
crictl pull nginx:latest
查看命名空间：
ctr ns list
ctr -n k8s.io images ls | grep nginx



# apt-transport-https 可能是一个虚拟包（dummy package）；如果是的话，你可以跳过安装这个包
sudo apt-get install -y apt-transport-https ca-certificates curl gpg


#  安装k8s
# 三台机器都执行
sudo apt-get update
# apt-transport-https 可能是一个虚拟包（dummy package）；如果是的话，你可以跳过安装这个包
sudo apt-get install -y apt-transport-https ca-certificates curl gpg




# 三台机器都执行
# 如果 `/etc/apt/keyrings` 目录不存在，则应在 curl 命令之前创建它，请阅读下面的注释。
# sudo mkdir -p -m 755 /etc/apt/keyrings
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg



# 三台机器都执行
# 此操作会覆盖 /etc/apt/sources.list.d/kubernetes.list 中现存的所有配置。
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list

apt update
# 查看可以安装的kubeadm版本、包信息、依赖信息等
apt-cache policy kubeadm
apt-cache showpkg kubeadm
apt-cache madison kubeadm

# 三台都执行
apt-get install -y kubelet kubeadm kubectl


# 锁定版本，防止自动更新   ，  三台都执行。
apt-mark hold kubelet kubeadm kubectl

# 三台机器都执行
vim /etc/default/kubelet
    # 添加内容: KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"

# 三台机器都执行
systemctl enable kubelet



# master节点执行, 此命令最终输出下一步操作命令，注意观察结果
kubeadm init --kubernetes-version=1.32.8 --control-plane-endpoint=k8smaster --apiserver-advertise-address=192.168.5.12 --pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/12 --image-repository=registry.aliyuncs.com/google_containers --upload-certs --v=9


# master节点执行
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config


# 其余worker节点执行
kubeadm join k8smaster:6443 --token zqv080.1q0ecziwmumhlnps --discovery-token-ca-cert-hash sha256:ebc241f081db52603d566c12f7e40bdeddb9dcabb341c3e663ce230b62f2ce49 


# 查看三个pod
kubectl get nodes

# 对etcd通信绕过代理  如果不通，可以试试
vi /etc/kubernetes/manifests/kube-apiserver.yaml
spec:
  containers:
  - name: kube-apiserver
    # 其他配置...
    env:
    - name: NO_PROXY
      value: "127.0.0.1,<etcd节点IP>,<集群内部CIDR>,.svc,.svc.cluster.local"
      # 补充集群内部常用地址（如Pod CIDR、Service CIDR、集群域名等）

 vi /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
    [Service]
    # 其他配置...
    Environment="NO_PROXY=127.0.0.1,<etcd节点IP>,<集群内部CIDR>,.svc,.svc.cluster.local"

sudo systemctl daemon-reload
sudo systemctl restart kubelet




# 处理网络， 在master执行
kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/calico.yaml
vim calico.yaml
  # 修改 ip网段 ， kubeadm init命令中的pod网段
  
# 查看pod
kubectl get pods -n kube-system

    
